package com.atguigu.edu.realtime220815.app.dim;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.atguigu.edu.realtime220815.app.func.DimSinkFunction;
import com.atguigu.edu.realtime220815.app.func.DimTableProcessFunction;
import com.atguigu.edu.realtime220815.bean.TableProcess;
import com.atguigu.edu.realtime220815.util.KafkaUtils;
import com.ververica.cdc.connectors.mysql.source.MySqlSource;
import com.ververica.cdc.connectors.mysql.table.StartupOptions;
import com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.state.MapStateDescriptor;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.BroadcastConnectedStream;
import org.apache.flink.streaming.api.datastream.BroadcastStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.util.Collector;

/**
 * @Classname DimApp
 * @Description TODO
 * @Date 2023/2/15 18:09
 * @Created by lzx
 */
public class DimApp {
    public static void main(String[] args) throws Exception {
        /*
        1.创建流式执行环境
        2.检查点设置
        3.从kafka topic_db中读取业务数据
        4.过滤,类型转换 Str -> JSONObject
        5.使用Flink CDC从mysql中读取维度配置表
        6.将配置流广播到所有并行度上
        7.将主流与广播流合并
        8.处理合并后的数据
        9.将处理好的数据写入Phoenix中
         */
        //1.创建流式执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(4);
        //2.检查点设置
        env.enableCheckpointing(5000, CheckpointingMode.EXACTLY_ONCE);
        env.setStateBackend(new HashMapStateBackend());
        env.setRestartStrategy(RestartStrategies.failureRateRestart(3, Time.days(30),Time.seconds(3)));
        env.getCheckpointConfig().setCheckpointTimeout(60000);
        env.getCheckpointConfig().setMinPauseBetweenCheckpoints(2000);
        env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
        env.getCheckpointConfig().setCheckpointStorage("hdfs://hadoop101:8020/eduRealTime/ck");
        System.setProperty("HADOOP_USER_NAME","atguigu");
        //3.从kafka topic_db中读取业务数据
        DataStreamSource<String> topicDbDs = env.addSource(KafkaUtils.getFlinkKafkaConsumer("topic_db", "dim_app_group"));

        //4.过滤,类型转换 Str -> JSONObject
        SingleOutputStreamOperator<JSONObject> process = topicDbDs.process(new ProcessFunction<String, JSONObject>() {
            @Override
            public void processElement(String value, ProcessFunction<String, JSONObject>.Context ctx, Collector<JSONObject> out) throws Exception {
                try {
                    JSONObject jsonObject = JSON.parseObject(value);
                    if ("bootstrap-start".equals(jsonObject.getString("type")) || "bootstrap-complete".equals(jsonObject.getString("type"))) {
                    }else {
                        out.collect(jsonObject);
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        });
        //process.print(">>>");
        //5.使用Flink CDC从mysql中读取维度配置表
        MySqlSource<String> mySqlSource = MySqlSource.<String>builder()
                .hostname("hadoop100")
                .port(3306)
                .databaseList("edu_config") // set captured database
                .tableList("edu_config.table_process") // set captured table
                .username("root")
                .password("000000")
                .deserializer(new JsonDebeziumDeserializationSchema()) // converts SourceRecord to JSON String
                .startupOptions(StartupOptions.initial())
                .build();
        DataStreamSource<String> configDS = env
                .fromSource(mySqlSource, WatermarkStrategy.noWatermarks(), "MySQL Source")
                .setParallelism(1);
        //configDS.print(">>>");

        //6.将配置流广播到所有并行度上
        MapStateDescriptor<String, TableProcess> configState = new MapStateDescriptor<>("config_state", String.class, TableProcess.class);
        BroadcastStream<String> configBroadStream = configDS.broadcast(configState);

        //7.将主流与广播流合并
        BroadcastConnectedStream<JSONObject, String> connect = process.connect(configBroadStream);
        //8.处理合并后的数据
        SingleOutputStreamOperator<JSONObject> streamOperator = connect.process(new DimTableProcessFunction(configState));
        //9.将处理好的数据写入Phoenix中
        streamOperator.addSink(new DimSinkFunction());

        env.execute();
    }
}
